package SMSGUI;

import java.awt.Image;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.StringTokenizer;

import SocialMediaSentiment.Perceptron;
import SocialMediaSentiment.ReadExcel;
import SocialMediaSentiment.StandardFunctions;
import SocialMediaSentiment.calcTrigramValues;
import SocialMediaSentiment.getNameFeatures;
import SocialMediaSentiment.levenstein2;

public class Functions {
	static boolean inLexicon = false;

	public static float[] calcBounderies(float startupper, float startlower)
			throws IOException {
		HashMap<String, Integer> trigrams = getTriGrams();
		Map<String, Double> woordenweightslijst = StandardFunctions
				.getWeightsList("WordsWeightsList.txt");
		float[] bounderies = new float[2];
		double besterror = 100;
		float stepsize = 0.01f;
		float bestupper = startupper;
		float bestlower = startlower;
		double uppererror, lowererror;

		besterror = calcTrigramValues.calcLimits(trigrams, bestupper,
				startlower, woordenweightslijst);
		while (stepsize < 1) {
			uppererror = calcTrigramValues.calcLimits(trigrams, bestupper
					+ stepsize, startlower, woordenweightslijst);
			lowererror = calcTrigramValues.calcLimits(trigrams, bestupper
					- stepsize, startlower, woordenweightslijst);
			if (uppererror < besterror && uppererror <= lowererror) {
				bestupper = bestupper + stepsize;
				besterror = uppererror;
				stepsize = 0.01f;
			} else if (lowererror < besterror && lowererror < uppererror) {
				bestupper = bestupper - stepsize;
				besterror = lowererror;
				stepsize = 0.01f;
			} else
				stepsize = stepsize * 2;
		}
		bounderies[0] = bestupper;
		stepsize = 0.01f;
		while (stepsize < 1) {
			uppererror = calcTrigramValues.calcLimits(trigrams, bestupper,
					bestlower + stepsize, woordenweightslijst);
			lowererror = calcTrigramValues.calcLimits(trigrams, bestupper,
					bestlower - stepsize, woordenweightslijst);
			if (uppererror < besterror && uppererror <= lowererror) {
				bestlower = bestlower + stepsize;
				besterror = uppererror;
				stepsize = 0.01f;
			} else if (lowererror < besterror && lowererror < uppererror) {
				bestlower = bestlower - stepsize;
				besterror = lowererror;
				stepsize = 0.01f;
			} else
				stepsize = stepsize * 2;
		}
		bounderies[1] = bestlower;

		return bounderies;

	}

	public static float[] calcBounderies(float startupper, float startlower,
			Map<String, Double> woordenweightslijst) throws IOException {
		HashMap<String, Integer> trigrams = getTriGrams();
		float[] bounderies = new float[3];
		float besterror = 100;
		float stepsize = 0.01f;
		float bestupper = startupper;
		float bestlower = startlower;
		float uppererror, lowererror;

		besterror = (float) calcTrigramValues.calcLimits(trigrams, bestupper,
				startlower, woordenweightslijst);
		// System.out.println("STARTBOUNDERIES: " + startupper + " " +
		// startlower);
		while (stepsize < 1) {
			uppererror = (float) calcTrigramValues.calcLimits(trigrams,
					bestupper + stepsize, startlower, woordenweightslijst);
			lowererror = (float) calcTrigramValues.calcLimits(trigrams,
					bestupper - stepsize, startlower, woordenweightslijst);
			if (uppererror < besterror && uppererror <= lowererror) {
				bestupper = bestupper + stepsize;
				besterror = uppererror;
				stepsize = 0.01f;
			} else if (lowererror < besterror && lowererror < uppererror) {
				bestupper = bestupper - stepsize;
				besterror = lowererror;
				stepsize = 0.01f;
			} else
				stepsize = stepsize * 2;
			// System.out.println("BESTUPPER AND ERROR UPDATED: " + bestupper +
			// " " + besterror + " stepsize: " + stepsize);
		}
		bounderies[0] = bestupper;
		stepsize = 0.01f;
		while (stepsize < 1) {
			uppererror = (float) calcTrigramValues.calcLimits(trigrams,
					bestupper, bestlower + stepsize, woordenweightslijst);
			lowererror = (float) calcTrigramValues.calcLimits(trigrams,
					bestupper, bestlower - stepsize, woordenweightslijst);
			if (uppererror < besterror && uppererror <= lowererror) {
				bestlower = bestlower + stepsize;
				besterror = uppererror;
				stepsize = 0.01f;
			} else if (lowererror < besterror && lowererror < uppererror) {
				bestlower = bestlower - stepsize;
				besterror = lowererror;
				stepsize = 0.01f;
			} else
				stepsize = stepsize * 2;
			// System.out.println("BESTLOWER AND ERROR UPDATED: " + bestlower +
			// " " + besterror + " stepsize: " + stepsize);
		}
		bounderies[1] = bestlower;
		bounderies[2] = besterror;
		return bounderies;

	}

	/**
	 * Calculate trigram value
	 * 
	 * @param weightlist
	 *            for the words
	 * @param word
	 *            1
	 * @param word
	 *            2
	 * @param word
	 *            3
	 * 
	 * @return trigram value
	 */
	private static double calculateTrigramValue(
			Map<String, Double> woordenweightslijst, String word1,
			String word2, String word3) {
		double weight1 = (woordenweightslijst.get(word1) != null) ? woordenweightslijst
				.get(word1) : 0;
		double weight2 = (woordenweightslijst.get(word2) != null) ? woordenweightslijst
				.get(word2) : 0;
		double weight3 = (woordenweightslijst.get(word3) != null) ? woordenweightslijst
				.get(word3) : 0;
		return (weight1 + weight2 + weight3) / 3;
	}

	static double CalculateTweetValue(Map<String, Double> woordenweightslijst,
			String tweet) {
		double tweetValue = 0.0;

		String trigrams = getTriGrams(tweet);
		String[] splittrigram = trigrams.split(",");

		for (int j = 0; j < splittrigram.length; j++) { // loop over all the
			// trigrams
			StringTokenizer tokenizer = new StringTokenizer(splittrigram[j],
					" []");
			if (tokenizer.countTokens() == 3) {
				String word1 = tokenizer.nextToken();
				String word2 = tokenizer.nextToken();
				String word3 = tokenizer.nextToken();

				tweetValue += calculateTrigramValue(woordenweightslijst, word1,
						word2, word3);
			}
		}
		tweetValue = tweetValue / splittrigram.length;
		return tweetValue;
	}
	
	static double CalculateTrigramValue(Map<String, Double> woordenweightslijst,
			String trigram) {
		double trigramValue = 0.0;
		StringTokenizer words = new StringTokenizer(trigram," !@#$%^&*()_+-=:;<>,./?");
		String word;
		double weight;
		while(words.hasMoreTokens()){
			word = words.nextToken();
			weight = (woordenweightslijst.get(word)!=null)?woordenweightslijst.get(word):0;
			//System.out.println(word +"  "+weight);
			
			trigramValue += weight;
			
		}
		
		return trigramValue;
	}
	

	static float CalculateTweetValueSentiment(
			Map<String, Double> woordenweightslijst, String tweet) {
		float tweetValue = 0.9f;

		String trigrams = getTriGrams(tweet);
		String[] splittrigram = trigrams.split(",");

		for (int j = 0; j < splittrigram.length; j++) { // loop over all the
			// trigrams
			StringTokenizer tokenizer = new StringTokenizer(splittrigram[j],
					" []");
			if (tokenizer.countTokens() == 3) {
				String word1 = tokenizer.nextToken();
				String word2 = tokenizer.nextToken();
				String word3 = tokenizer.nextToken();

				tweetValue += (float) calculateTrigramValue(
						woordenweightslijst, word1, word2, word3);
			}
		}
		tweetValue = tweetValue / splittrigram.length;

		return tweetValue;
	}

	public static double convertExpectedTweetValueToClass(double tweetValue) {
		return 1;
	}

	public static int convertExpectedTweetValueToClass(float tweetValue,
			float boundery) {
		if (tweetValue >= boundery) {
			return 1;
		}

		return 0;
	}

	public static double convertExpectedTweetValueToClass(double tweetValue,
			float[] bounderies) {
		bounderies[0] *= 2;
		bounderies[1] *= 2;
		if (tweetValue > bounderies[0]) {
			// System.out.println(tweetValue+" "+bounderies[0]+" 1");
			return 1;
		}

		if (tweetValue < bounderies[1]) {
			// System.out.println(tweetValue+" "+bounderies[1]+" -1");
			return -1;
		}
		// System.out.println(tweetValue+" 0");
		return 0;
	}

	/**
	 * Convert the Calculated tweet value to Class
	 * 
	 * @param tweetvalue
	 *            Value of the tweet(double)
	 * @param boundery
	 *            boundery between no sentiment and sentiment
	 * 
	 * @return 0 if tweet has no sentiment 1 if tweet has sentiment
	 */
	public static float convertExpectedTweetValueToClassSentiment(
			double tweetValue, float boundery) {

		if (tweetValue > boundery) {
			System.out.println("expected: 1");
			return 1;
		}
		System.out.println("expected: 0");
		return 0;
	}

	public static String CorrectTweet(String tweet) {

		File input = new File("sortedLexicon1.txt");
		String lexicon = StandardFunctions.deserializeString(input);

		String bestTweetWord, currentToken, currentWord, correctedTweet = "";
		int d;
		double distance, bestDistance;

		StringTokenizer st = new StringTokenizer(lexicon,
				" \r\n\t\f,.!$%^&*()[]{}-=\\;<>/?");
		StringTokenizer words = new StringTokenizer(tweet,
				" \r\n\t\f,.!$%^&*()[]{}-=\\;<>/?");

		while (words.hasMoreTokens()) {
			// pak tweetwoord en reset
			currentWord = words.nextToken();
			bestTweetWord = currentWord;
			bestDistance = 1000;
			st = new StringTokenizer(lexicon);

			while (st.hasMoreTokens()) {
				// pak nieuw woord uit lexicon en reset distance
				currentToken = st.nextToken();

				// bereken levenshtein distance
				d = levenstein2.levenshteinDistance(currentWord, currentToken);
				distance = (double) d / currentWord.length();

				// als distance = 0, write to file en break
				if (distance < bestDistance && distance < 0.25) {
					bestDistance = distance;
					bestTweetWord = currentToken;
				}

				// als de distance kleiner is dan de beste distance update,
				// distance en woord
				if (d == 0) {
					while (st.hasMoreTokens()) {
						st.nextToken();
					}
				}
			}
			correctedTweet += " " + bestTweetWord;

		}
		return correctedTweet;
	}

	public static String CorrectWord(String word) {

		String bestTweetWord, currentToken;
		int d;
		double distance, bestDistance;
		File input = new File("sortedLexicon1.txt");
		String lexicon = StandardFunctions.deserializeString(input);
		StringTokenizer stLexicon = new StringTokenizer(lexicon," \r\n\t\f,.!$%^&*()[]{}-=\\;<>/?");

		bestTweetWord = word;
		bestDistance = 1000;
		inLexicon = false;

		while (stLexicon.hasMoreTokens()) {

			// pak nieuw woord uit lexicon en reset distance
			currentToken = stLexicon.nextToken();

			// bereken levenshtein distance
			d = levenstein2.levenshteinDistance(word, currentToken);
			distance = (double) d / word.length();

			// als distance = 0, write to file en break
			if (distance < bestDistance && distance < 0.25) {
				bestDistance = distance;
				bestTweetWord = currentToken;
			}

			// als de distance kleiner is dan de beste distance update,
			// distance en woord
			if (d == 0) {
				inLexicon = true;
				while (stLexicon.hasMoreTokens()) {
					stLexicon.nextToken();
				}
			}
		}
		return bestTweetWord;
	}

	public static String deserializeString(File file) {
		try {
			int len;
			char[] chr = new char[4096];
			final StringBuffer buffer = new StringBuffer();
			final FileReader reader = new FileReader(file);
			try {
				while ((len = reader.read(chr)) > 0) {
					buffer.append(chr, 0, len);
				}
			} finally {
				reader.close();
			}
			return buffer.toString();

		} catch (IOException e) {
			System.err.println("Exception ");
			return null;
		}

	}

	static boolean getInLexicon() {
		return inLexicon;
	}

	public static HashMap<String, Integer> getTriGrams() {
		File file = new File("TrainSetAllWordsTweets.txt");
		StringBuffer contents = new StringBuffer();
		BufferedReader reader = null;

		try {
			reader = new BufferedReader(new FileReader(file));
			String text = null;

			// repeat until all lines is read
			while ((text = reader.readLine()) != null) {
				contents.append(text).append(
						System.getProperty("line.separator"));
			}
		} catch (FileNotFoundException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		} finally {
			try {
				if (reader != null) {
					reader.close();
				}
			} catch (IOException e) {
				e.printStackTrace();
			}
		}
		char[] chararray = new char[contents.length()];
		contents.getChars(0, contents.length(), chararray, 0);

		StringTokenizer st = new StringTokenizer(contents.toString(),
				"\r\n\t\f,.!$%^&*()[]{}-=\\;<>/?");
		String word = null;
		ArrayList<String> tweets = new ArrayList<String>();
		ArrayList<Integer> sentiment = new ArrayList<Integer>();
		StringBuffer tweet = new StringBuffer();
		int number = st.countTokens();
		for (int counter = 0; number - counter > 0; counter++) {

			word = st.nextToken();
			if (word.contains("NEXTTWEEEEET")) {
				// add all words to new tweed
				int sentimentvalue;

				if (word.length() == 13) // positief of 0
					sentimentvalue = Character.getNumericValue(word.charAt(0));
				else {
					sentimentvalue = -1
							* Character.getNumericValue(word.charAt(3));
				}
				tweets.add(tweet.toString());
				sentiment.add(sentimentvalue);
				// remove tweet
				tweet.delete(0, tweet.length());
			} else {
				// add word to tweet
				tweet.append(word + " ");
			}
		}

		ArrayList<String> ngrams = new ArrayList<String>();
		HashMap<String, Integer> allNGrams = new HashMap<String, Integer>();
		StringTokenizer messageNGram;
		StringTokenizer messageNGram2;
		StringTokenizer messageNGram3;
		// For each message, make the ngrams. (tri-grams)
		for (String message : tweets) {
			int index = tweets.indexOf(message);
			messageNGram = new StringTokenizer(message, " ");
			messageNGram2 = new StringTokenizer(message, " ");
			messageNGram2.nextToken();
			messageNGram3 = new StringTokenizer(message, " ");
			messageNGram3.nextToken();
			messageNGram3.nextToken();
			String part1, part2, part3;
			number = messageNGram.countTokens();
			String trigram = "";
			for (int i = 0; number - i > 2; i++) {
				part1 = messageNGram.hasMoreTokens() ? messageNGram.nextToken()
						: "";
				part2 = messageNGram2.hasMoreTokens() ? messageNGram2
						.nextToken() : "";
				part3 = messageNGram3.hasMoreTokens() ? messageNGram3
						.nextToken() : "";
				trigram = part1 + " " + part2 + " " + part3;
				if (!trigram.isEmpty()) {
					ngrams.add(trigram);
				}
			}
			allNGrams.put(message, sentiment.get(index));
			ngrams.clear();
		}
		return allNGrams;
	}

	private static String getTriGrams(String message) {
		StringTokenizer messageNGram;
		StringTokenizer messageNGram2;
		StringTokenizer messageNGram3;
		messageNGram = new StringTokenizer(message, " ");
		messageNGram2 = new StringTokenizer(message, " ");
		messageNGram2.nextToken();
		messageNGram3 = new StringTokenizer(message, " ");
		messageNGram3.nextToken();
		messageNGram3.nextToken();
		String part1, part2, part3;
		int number = messageNGram.countTokens();
		String trigram = "";
		String ngrams = "";

		for (int i = 0; number - i > 2; i++) {
			part1 = messageNGram.hasMoreTokens() ? messageNGram.nextToken()
					: "";
			part2 = messageNGram2.hasMoreTokens() ? messageNGram2.nextToken()
					: "";
			part3 = messageNGram3.hasMoreTokens() ? messageNGram3.nextToken()
					: "";
			trigram = (number - i > 3) ? part1 + " " + part2 + " " + part3
					+ ", " : part1 + " " + part2 + " " + part3;

			ngrams += trigram;

		}
		return ngrams;
	}

	public static Map<String, Double> getTweetWeights(String tweet,
			Map<String, Double> weightlist) {
		Map<String, Double> woordenWeights = new HashMap<String, Double>();
		StringTokenizer st = new StringTokenizer(tweet,
				" \r\n\t\f,.!$%^&*()[]{}-=\\;<>/?:_");
		String word;
		// get initial weights of words
		while (st.hasMoreTokens()) {
			word = st.nextToken();

			// look up weight
			if (word.length() > 2) {
				Double weight = (weightlist.get(word) != null) ? weightlist
						.get(word) : 0;
				woordenWeights.put(word, weight);
			}

		}
		return woordenWeights;
	}
	
	public static Map<String, Integer> getCountList(String fileName) 
	{       
		File file ;
		HashMap<Integer, String[]> woordenlijstSentiment;
		Map<String, Integer> woordencountlijst = new HashMap<String, Integer>();
		String[] nameExtention;
		nameExtention = fileName.split("\\.");
		if (nameExtention.length!=2){
			System.err.println("Could not recognize file type");
		}
		switch(nameExtention[1]){
		case "txt": 
			file = new File(fileName);
			StringBuffer contents = new StringBuffer();
			BufferedReader reader = null;

			Map<String, Integer> countlijst = new HashMap<String, Integer>();
			int value = 0;

			try {
				reader = new BufferedReader(new FileReader(file));
				String text = null;

				// repeat until all lines is read
				while ((text = reader.readLine()) != null) {
					contents.append(text).append(System.getProperty("line.separator"));
				}
			} catch (FileNotFoundException e) {
				e.printStackTrace();
			} catch (IOException e) {
				e.printStackTrace();
			} finally {
				try {
					if (reader != null) {
						reader.close();
					}
				} catch (IOException e) {
					e.printStackTrace();
				}
			}
			char[] chararray = new char[contents.length()];
			contents.getChars(0, contents.length(), chararray, 0);
			String key;

			StringTokenizer words = new StringTokenizer(contents.toString(), "\n?");

			while (words.hasMoreTokens()) 
			{
				StringTokenizer wordweight = new StringTokenizer(words.nextToken(), " \n\t\r\f");
				key = wordweight.nextToken();    
				if(key.length()>1){
					String stvalue = wordweight.nextToken();
					value = Integer.parseInt(stvalue); 
					countlijst.put(key, value);
				}

			}
			return countlijst;
		case "xls":
			try {
				woordenlijstSentiment = ReadExcel.read(fileName, 0,1);

				String[] woordenlijstSentimentData;
				String word;
				int weight;
				Iterator<Integer> k = woordenlijstSentiment.keySet().iterator();
				while(k.hasNext()){
					woordenlijstSentimentData = woordenlijstSentiment.get(k.next());
					word = woordenlijstSentimentData[0];
					weight = (1-Integer.parseInt(woordenlijstSentimentData[1].replace(",", ".")))/2;
					woordencountlijst.put(word, weight);
				}
			} catch (IOException e1) {
				e1.printStackTrace();
							}
			return woordencountlijst;
		default:
			file = new File("WordsWeightsList.txt");
			break;
		}
		return null;
	}

	 /**
     * Get the weightlist from file
     *
     * @param 0	WordsWeightsList.txt
     * @param 1	updatedWordsWeightsList.txt
     * @param 2 WordsWeightsListSentiment.txt
     * @param 3 updatedWordsWeightsListSentiment.txt
     * @param 4 CompareZeroNonZero.xls
     * @param 5 CompareZeroNonZero.xls weights initialized at 0
     * 
     * @return key of the Map is the word. The value is the weight of that word
     */
	public static Map<String, Double> getWeightsListOLD(int updated)
	{       
		File file ;
		HashMap<Integer, String[]> woordenlijstSentiment;
		HashMap<String, Double> woordenweightslijst = new HashMap<String, Double>();

		switch (updated) {
		case 0:
			file = new File("WordsWeightsList.txt");
			break;
		case 1:
			file = new File("updatedWordsWeightsList.txt");
			break;
		case 2:
			file = new File("WordsWeightsListSentiment.txt");
			break;
		case 3:
			file = new File("updatedWordsWeightsListSentiment.txt");
			break;
		case 4:
			try {
				woordenlijstSentiment = ReadExcel.read(
						"CompareZeroNonZero.xls", 0, 1);

				String[] woordenlijstSentimentData;
				String word;
				double weight;
				Iterator<Integer> k = woordenlijstSentiment.keySet().iterator();
				while (k.hasNext()) {
					woordenlijstSentimentData = woordenlijstSentiment.get(k
							.next());
					word = woordenlijstSentimentData[0];
					weight = (1 - Double
							.parseDouble(woordenlijstSentimentData[1].replace(
									",", "."))) / 2;
					woordenweightslijst.put(word, weight);
				}
			} catch (IOException e1) {
				// TODO Auto-generated catch block
				e1.printStackTrace();
			}
			return woordenweightslijst;
		case 5:

			try {
				woordenlijstSentiment = ReadExcel.read(
						"CompareZeroNonZero.xls", 0, 1);

				String[] woordenlijstSentimentData;
				String word;
				double weight;
				Iterator<Integer> k = woordenlijstSentiment.keySet().iterator();
				while (k.hasNext()) {
					woordenlijstSentimentData = woordenlijstSentiment.get(k
							.next());
					word = woordenlijstSentimentData[0];
					weight = 0;
					woordenweightslijst.put(word, weight);
				}
			} catch (IOException e1) {
				// TODO Auto-generated catch block
				e1.printStackTrace();
			}
			return woordenweightslijst;
		default:
			file = new File("WordsWeightsList.txt");
			break;
		}

		StringBuffer contents = new StringBuffer();
		BufferedReader reader = null;

		Map<String, Double> weightlijst = new HashMap<String, Double>();
		double value = 0;

		try {
			reader = new BufferedReader(new FileReader(file));
			String text = null;

			// repeat until all lines is read
			while ((text = reader.readLine()) != null) {
				contents.append(text).append(
						System.getProperty("line.separator"));
			}
		} catch (FileNotFoundException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		} finally {
			try {
				if (reader != null) {
					reader.close();
				}
			} catch (IOException e) {
				e.printStackTrace();
			}
		}
		char[] chararray = new char[contents.length()];
		contents.getChars(0, contents.length(), chararray, 0);
		String key;

		StringTokenizer words = new StringTokenizer(contents.toString(), "\n?");

		while (words.hasMoreTokens()) {
			StringTokenizer wordweight = new StringTokenizer(words.nextToken(),
					" ");
			key = wordweight.nextToken();
			if (key.length() > 1) {
				value = Double.parseDouble(wordweight.nextToken());
				weightlijst.put(key, value);
			}

		}
		return weightlijst;
	}

	/**
	 * Get the weightlist from file
	 * 
	 * @param file
	 *            name inclucing extentions. xls and txt supported
	 * 
	 * @return key of the Map is the word. The value is the weight of that word
	 */
	public static Map<String, Double> getWeightsList(String fileName) {
		File file;
		HashMap<Integer, String[]> woordenlijstSentiment;
		HashMap<String, Double> woordenweightslijst = new HashMap<String, Double>();
		String[] nameExtention;
		nameExtention = fileName.split("\\.");
		if (nameExtention.length > 2) {
			System.err.println("Could not recognize file type");
		}
		switch (nameExtention[1]) {
		case "txt":
			file = new File(fileName);
			StringBuffer contents = new StringBuffer();
			BufferedReader reader = null;

			Map<String, Double> weightlijst = new HashMap<String, Double>();
			double value = 0;

			try {
				reader = new BufferedReader(new FileReader(file));
				String text = null;

				// repeat until all lines is read
				while ((text = reader.readLine()) != null) {
					contents.append(text).append(
							System.getProperty("line.separator"));
				}
			} catch (FileNotFoundException e) {
				e.printStackTrace();
			} catch (IOException e) {
				e.printStackTrace();
			} finally {
				try {
					if (reader != null) {
						reader.close();
					}
				} catch (IOException e) {
					e.printStackTrace();
				}
			}
			char[] chararray = new char[contents.length()];
			contents.getChars(0, contents.length(), chararray, 0);
			String key;

			StringTokenizer words = new StringTokenizer(contents.toString(),
					"\n?");

			while (words.hasMoreTokens()) {
				StringTokenizer wordweight = new StringTokenizer(
						words.nextToken(), " ");
				key = wordweight.nextToken();
				if (key.length() > 1) {
					value = Double.parseDouble(wordweight.nextToken());
					weightlijst.put(key, value);
				}

			}
			return weightlijst;
		case "xls":
			try {
				woordenlijstSentiment = ReadExcel.read(fileName, 0, 1);

				String[] woordenlijstSentimentData;
				String word;
				double weight;
				Iterator<Integer> k = woordenlijstSentiment.keySet().iterator();
				while (k.hasNext()) {
					woordenlijstSentimentData = woordenlijstSentiment.get(k
							.next());
					word = woordenlijstSentimentData[0];
					weight = (1 - Double
							.parseDouble(woordenlijstSentimentData[1].replace(
									",", "."))) / 2;
					woordenweightslijst.put(word, weight);
				}
			} catch (IOException e1) {
				e1.printStackTrace();
			}
			return woordenweightslijst;
		default:
			file = new File("WordsWeightsList.txt");
			break;
		}
		return null;
	}
	
	 
	/**
	 * Get the weightlist from file
	 * 
	 * @param filename
	 *            inclucing extentions. xls and txt supported
	 * @param the
	 *            weight you want all the words to have.
	 * 
	 * @return key of the Map is the word. The value is the weight of that word
	 */
	public static Map<String, Double> getWeightsList(String fileName,
			double initialize) {
		File file;
		HashMap<Integer, String[]> woordenlijstSentiment;
		HashMap<String, Double> woordenweightslijst = new HashMap<String, Double>();
		String[] nameExtention = fileName.split("\\.");
		if (nameExtention.length > 2) {
			System.err.println("Could not recognize file type");
		}
		switch (nameExtention[1]) {
		case "txt":
			file = new File(fileName);
			StringBuffer contents = new StringBuffer();
			BufferedReader reader = null;

			Map<String, Double> weightlijst = new HashMap<String, Double>();
			double value = 0;

			try {
				reader = new BufferedReader(new FileReader(file));
				String text = null;

				// repeat until all lines is read
				while ((text = reader.readLine()) != null) {
					contents.append(text).append(
							System.getProperty("line.separator"));
				}
			} catch (FileNotFoundException e) {
				e.printStackTrace();
			} catch (IOException e) {
				e.printStackTrace();
			} finally {
				try {
					if (reader != null) {
						reader.close();
					}
				} catch (IOException e) {
					e.printStackTrace();
				}
			}
			char[] chararray = new char[contents.length()];
			contents.getChars(0, contents.length(), chararray, 0);
			String key;

			StringTokenizer words = new StringTokenizer(contents.toString(),
					"\n?");

			while (words.hasMoreTokens()) {
				StringTokenizer wordweight = new StringTokenizer(
						words.nextToken(), " ");
				key = wordweight.nextToken();
				if (key.length() > 1) {
					value = initialize;
					weightlijst.put(key, value);
				}

			}
			return weightlijst;
		case "xls":
			try {
				woordenlijstSentiment = ReadExcel.read(fileName, 0, 1);

				String[] woordenlijstSentimentData;
				String word;
				double weight;
				Iterator<Integer> k = woordenlijstSentiment.keySet().iterator();
				while (k.hasNext()) {
					woordenlijstSentimentData = woordenlijstSentiment.get(k
							.next());
					word = woordenlijstSentimentData[0];
					weight = initialize;
					woordenweightslijst.put(word, weight);
				}
			} catch (IOException e1) {
				e1.printStackTrace();
			}
			return woordenweightslijst;
		default:
			file = new File("WordsWeightsList.txt");
			break;
		}
		return null;
	}

	@SuppressWarnings("rawtypes")
	public static void HashMapToFile(HashMap map, String filename)
			throws IOException {
		FileWriter outFile = new FileWriter(filename);
		PrintWriter out = new PrintWriter(outFile);
		Iterator k = map.keySet().iterator();
		while (k.hasNext()) {
			Object key = k.hasNext();
			Object value = map.get(key);
			out.println(key + " " + value);
		}
		out.close();
	}

	public static Map<String, Double> makeWeightsList() throws IOException {
		HashMap<String, Double> comparePosNeg = ReadExcel
				.readCompareMaps("comparePositiefNegatief.xls");
		HashMap<String, Double> compareZeroNonZero = ReadExcel
				.readCompareMaps("CompareZeroNonZero.xls");
		Map<String, Double> woordenweightslijst = new HashMap<String, Double>();
		Double weight;

		for (String key : compareZeroNonZero.keySet()) {
			weight = (1 - compareZeroNonZero.get(key)) / 2;
			weight = (weight != 0) ? weight * comparePosNeg.get(key) : weight;
			woordenweightslijst.put(key, weight);
		}

		return woordenweightslijst;
	}

	public static Map<String, Double> scaleWeightList(
			Map<String, Double> woordenweightslijst) {

		Iterator<Double> z = woordenweightslijst.values().iterator();
		double largestWeight = 0, smallestWeight = 0, unscaledWeight, largestDiff;
		String word;

		while (z.hasNext()) {
			unscaledWeight = z.next();
			largestWeight = (unscaledWeight > largestWeight) ? unscaledWeight
					: largestWeight;
			smallestWeight = (unscaledWeight < smallestWeight) ? unscaledWeight
					: smallestWeight;
		}

		largestDiff = (largestWeight > Math.abs(smallestWeight)) ? largestWeight
				: Math.abs(smallestWeight);

		Iterator<String> x = woordenweightslijst.keySet().iterator();
		while (x.hasNext()) {
			word = x.next();
			woordenweightslijst.put(word, woordenweightslijst.get(word)
					/ largestDiff);
		}
		return woordenweightslijst;
	}

	public static int TweetClassify(String tweet, float boundery,
			Map<String, Double> woordenweightslijst) {
		Map<String, Double> tweetWeigths = getTweetWeights(tweet,
				woordenweightslijst);
		float tweetValue = (float) CalculateTweetValue(tweetWeigths, tweet);
		int tweetClass = convertExpectedTweetValueToClass(tweetValue, boundery);
		return tweetClass;
	}
	/**
	 * @param tweet
	 * @param boundery
	 * @param woordenweightslijst
	 * @return
	 */
	public static Map<Integer, HashMap<String, Double>> TweetClassifyData(String tweet, float boundery, Map<String, Double> woordenweightslijst ){
		Map<String, Double> tweetWeigths = getTweetWeights(tweet, woordenweightslijst);
		Iterator k = tweetWeigths.keySet().iterator();
		String word;
		double weight;
		//System.out.println(tweet);
		while(k.hasNext()){
			word = (String) k.next();
			weight = tweetWeigths.get(word);
			//System.out.printf("%s \t %f\n",word,weight);
			
		}
		float tweetValue = (float) CalculateTweetValue(tweetWeigths, tweet);
		int tweetClass = convertExpectedTweetValueToClass(tweetValue, boundery);
		Map<Integer, HashMap<String, Double>> tweetData= new HashMap<Integer,HashMap<String,Double>>();
		tweetData.put(tweetClass,(HashMap<String, Double>) tweetWeigths);
		return tweetData;
	}

	public static void writeSentimentsToFile(Map<String, Integer> weights) {

		try {
			// Create file
			FileWriter fstream = new FileWriter("CorrectedTweets.txt");
			BufferedWriter out = new BufferedWriter(fstream);
			for (String keys : weights.keySet()) {
				out.write(keys + " " + weights.get(keys));
				out.newLine();
			}
			// Close the output stream
			out.close();
		} catch (Exception e) {// Catch exception if any
			System.err.println("Error: " + e.getMessage());
		}
	}

	public static void writeWeigthsToFile(Map<String, Double> weights,
			String file) {

		try {
			// Create file
			// FileWriter fstream = new
			// FileWriter("updatedWordsWeightsList.txt");
			FileWriter fstream = new FileWriter(file);

			BufferedWriter out = new BufferedWriter(fstream);
			for (String keys : weights.keySet()) {
				out.write(keys + " " + weights.get(keys));
				out.newLine();
			}
			// Close the output stream
			out.close();
		} catch (Exception e) {// Catch exception if any
			System.err.println("Error: " + e.getMessage());
		}
	}

	public static HashMap<String, List<String>> getWoordenlijsten() {
		HashMap<String, List<String>> woordenlijsten = new HashMap<String, List<String>>();
		woordenlijsten.put("+", getWoordenlijst("positievenLijst.txt"));
		woordenlijsten.put("-", getWoordenlijst("negatievenLijst.txt"));
		woordenlijsten.put("v", getWoordenlijst("versterkingLijst.txt"));
		woordenlijsten.put("o", getWoordenlijst("ontkenningLijst.txt"));
		return woordenlijsten;

	}

	public static List<String> getWoordenlijst(String file) {
		List<String> woordenlijst = new ArrayList<String>();
		File input = new File(file);
		String woorden = StandardFunctions.deserializeString(input);
		StringTokenizer stLexicon = new StringTokenizer(woorden,
				" \r\n\t\f,.!$%^&*()[]{}-=\\;<>/?#");
		while (stLexicon.hasMoreTokens()) {
			woordenlijst.add(stLexicon.nextToken());
		}
		return woordenlijst;
	}

	static List<String[]> getTriGramsList(String tweet) {
		tweet = "NEUTRALWORD NEUTRALWORD " + tweet + " NEUTRALWORD NEUTRALWORD";
		StringTokenizer messageNGram = new StringTokenizer(tweet, " ");
		StringTokenizer messageNGram2 = new StringTokenizer(tweet, " ");
		StringTokenizer messageNGram3 = new StringTokenizer(tweet, " ");
		messageNGram2.nextToken();
		messageNGram3.nextToken();
		messageNGram3.nextToken();
		String[] trigram = new String[3];
		List<String[]> trigrams = new ArrayList<String[]>();

		while (messageNGram3.hasMoreTokens()) {
			trigram = new String[3];
			trigram[0] = messageNGram.nextToken();
			trigram[1] = messageNGram2.nextToken();
			trigram[2] = messageNGram3.nextToken();
			trigrams.add(trigram);
		}

		return trigrams;
	}

	public static String deterimeTrigramStructures(String[] trigram,
			HashMap<String, List<String>> woordSoort) {
		List<String> words;
		String wordType;
		String classification;
		boolean foundWord = false;
		String[] classi = new String[3];
		for (int i = 0; i < trigram.length; i++) {
			foundWord = false;
			Iterator<String> k = woordSoort.keySet().iterator();
			while (k.hasNext()) {
				wordType = k.next();
				words = woordSoort.get(wordType);
				if(trigram[i].equals("maar"))
					classi[i] = "RESET";
				if(trigram[0].contains("response"))//first word in trigram is reponse then the tweet is over
					return "STOP";
				if (words.contains(trigram[i].replace("#", ""))) {
					classi[i] = wordType;
					foundWord = true;
					break;
				}

			}
			if(!foundWord)
				classi[i] = "N";

		}
		//N = neutraal | + = positief | - = negatief | o = ontkenning | v = versterking
			if(classi[0].equals("-") && classi[1].equals("+") && classi[2].equals("N"))
				classification = "-";
			else if(classi[0].equals("N") && classi[1].equals("-") && classi[2].equals("+"))
				classification = "-";
			else if(classi[0].equals("RESET"))
				classification = "RESET";
			else if(classi[0].equals("+") && classi[1].equals("-") && classi[2].equals("N"))
				classification = "-";
			else if(classi[0].equals("N") && classi[1].equals("+") && classi[2].equals("-"))
				classification = "-";
			else if(classi[0].equals("v") && classi[1].equals("-") && classi[2].equals("N"))
				classification = "---";
			else if(classi[0].equals("N") && classi[1].equals("v") && classi[2].equals("-"))
				classification = "---";
			else if(classi[0].equals("v") && classi[1].equals("+") && classi[2].equals("N"))
				classification = "+++";
			else if(classi[0].equals("N") && classi[1].equals("v") && classi[2].equals("+"))
				classification = "+++";
			else if(classi[0].equals("v") && classi[1].equals("+") && classi[2].equals("-"))
				classification = "---";
			else if(classi[0].equals("v") && classi[1].equals("o") )
				classification = "-";
			else if(classi[1].equals("v") && classi[2].equals("o") )
				classification = "-";
			else if(classi[0].equals("o") && classi[1].equals("+") && classi[2].equals("N"))
				classification = "o+";
			else if(classi[0].equals("N") && classi[1].equals("o") && classi[2].equals("+"))
				classification = "o+";
			else if(classi[0].equals("o") && classi[1].equals("-") && classi[2].equals("N"))
				classification = "o-";
			else if(classi[0].equals("N") && classi[1].equals("o") && classi[2].equals("-"))
				classification = "o-";
			else if(classi[0].equals("+") && classi[1].equals("N") && classi[2].equals("N"))
				classification = "+";
			else if(classi[0].equals("N") && classi[1].equals("+") && classi[2].equals("N"))
				classification = "+";
			else if(classi[0].equals("N") && classi[1].equals("N") && classi[2].equals("+"))
				classification = "+";
			else if(classi[0].equals("+") && classi[1].equals("+") && classi[2].equals("N"))
				classification = "++";
			else if(classi[0].equals("+") && classi[1].equals("N") && classi[2].equals("+"))
				classification = "++";
			else if(classi[0].equals("N") && classi[1].equals("+") && classi[2].equals("+"))
				classification = "++";			
			else if(classi[0].equals("-") && classi[1].equals("N") && classi[2].equals("N"))
				classification = "-";
			else if(classi[0].equals("N") && classi[1].equals("-") && classi[2].equals("N"))
				classification = "-";
			else if(classi[0].equals("N") && classi[1].equals("N") && classi[2].equals("-"))
				classification = "-";
			else if(classi[0].equals("-") && classi[1].equals("-") && classi[2].equals("N"))
				classification = "--";
			else if(classi[0].equals("-") && classi[1].equals("N") && classi[2].equals("-"))
				classification = "--";
			else if(classi[0].equals("N") && classi[1].equals("-") && classi[2].equals("-"))
				classification = "--";
			else if(classi[0].equals("-") && classi[1].equals("-") && classi[2].equals("-"))
				classification = "---";
			else if(classi[0].equals("+") && classi[1].equals("+") && classi[2].equals("+"))
				classification = "+++";
			else if(classi[0].equals("o") && classi[1].equals("N") && classi[2].equals("-"))
				classification = "o-";
			else if(classi[0].equals("o") && classi[1].equals("N") && classi[2].equals("+"))
				classification = "o+";
			else if(classi[0].equals("o") && classi[1].equals("N") && classi[2].equals("N"))
				classification = "o";
			else if(classi[0].equals("N") && classi[1].equals("o") && classi[2].equals("N"))
				classification = "o";
			else if(classi[0].equals("N") && classi[1].equals("N") && classi[2].equals("o"))
				classification = "o";
			else if(classi[0].equals("+") && classi[1].equals("o") && classi[2].equals("N"))
				classification = "+";
			else if(classi[0].equals("-") && classi[1].equals("o") && classi[2].equals("N"))
				classification = "-";
			else if(classi[0].equals("+") && classi[1].equals("N") && classi[2].equals("o"))
				classification = "+o";
			else if(classi[0].equals("-") && classi[1].equals("N") && classi[2].equals("o"))
				classification = "-o";
			else
				classification = "N";
				
		
		
		return classification;
	}
	
	

	public static void overEO(HashMap<String, Integer> tweets, double alpha) {
		Map<String, Double> weightlist = getWeightsList("updatedWordsWeightsList.txt");
		String trigrams,trigram,tweet,word;
		HashMap<String, Integer> trigramList = new HashMap<String, Integer>();
		int sentiment;
		int tweetClass;
		int fouten = 0,correct = 0;
		File input = new File("boundery.txt");
		float boundery = new Float(StandardFunctions.deserializeString(input));
		Iterator<String> ktweets = tweets.keySet().iterator();
		
		while(ktweets.hasNext()){
			tweet = ktweets.next();
			sentiment = (tweets.get(tweet)!=0)?1:0;
			trigrams = getTriGrams(tweet);
			StringTokenizer trigramtokens = new StringTokenizer(trigrams, ",");
			//System.out.println(tweet);
			while(trigramtokens.hasMoreElements()){
				trigram = trigramtokens.nextToken();
				if(trigram.toLowerCase().contains("eo")&& voorzetsel(trigram)){
					tweetClass = TweetClassify(tweet,boundery,weightlist);
					System.out.printf("%s \n%s \t - %d - %d\n",tweet,trigram, sentiment,tweetClass);
					if(sentiment!=tweetClass){
						fouten++;
					}else{
						correct++;
					}
					
					//trigramList.put(trigram,sentiment);
					
					break;
				}
					
				
//				if(trigram.toLowerCase().contains("eo")){
//					//System.out.printf("%s \t - %d\n",trigram,sentiment);
//					trigramList.put(trigram,sentiment);
//				}
				
				
				
			}
			
		}
		//Perceptron.PerceptronTrigramOverEo(trigramList, 0.01);
		System.out.printf("fouten %d correct %d total %d", fouten, correct, fouten+correct);
		
		
	}

	private static boolean voorzetsel(String trigram) {
		String voorzetsels = "door van bij over met volgens tegen #eo eoeva eomagazine";
		StringTokenizer st = new StringTokenizer(voorzetsels," ");
		while(st.hasMoreTokens()){
			if (trigram.contains(st.nextToken()))
				return true;
		}
		
		return false;
	}


	
	/**
	 * @return
	 */
	public static Image tweetLocator(){
		Image image = null;
		return image;
	}

	public static float[] genderFinder(HashMap<Integer, String[]> namegessent) throws IOException {
		HashMap<String, String> nameges = getNameFeatures.getNamesGeslacht();
		float[] sentiment = getNameFeatures.compareGeslachtandNames(nameges, namegessent );	
		return sentiment;
	}

	
	/**
	 * 
	 * @param tweetSentimentMap Integer = index, String[] = {tweet, sentiment} sentiment is initialized as null
	 * @return
	 * @throws IOException 
	 */
	public static HashMap<Integer, Integer> classify(HashMap<Integer, String> tweetMap) throws IOException {
		//NaiveBayes.NaiveBayes(tweetMap);
		return NaiveBayes.NaiveBayes(tweetMap);
	}
	
	public static HashMap<Integer, String> correctTweets(HashMap<Integer, String> tweetMap){
		HashMap <Integer, String> tweets = new HashMap<Integer, String>();
		String tweet;
		
		Iterator<Integer> k = tweetMap.keySet().iterator();
		double amountOftweets = tweetMap.size(),done = 1;
		int index;
		String data;
		while(k.hasNext()){
			index = k.next();
			System.out.println(done+"\t"+(double)(done/amountOftweets)*100);
			done++;
			data = tweetMap.get(index);
			tweet = data;
			System.out.println(tweet);
			tweet = CorrectTweet(tweet);			
			System.out.println(tweet);
			tweets.put( index, tweet );
		}
	
		
		return tweets;
	}
	
	
}
